{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  第六步：降低学习率，调整树的数目"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-04T04:45:09.140433Z",
     "start_time": "2018-01-04T04:45:07.651575Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 读取数据 & 数据探索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-04T04:45:11.654248Z",
     "start_time": "2018-01-04T04:45:10.400421Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dpath = './data/'\n",
    "data = pd.read_csv(dpath + 'RentListingInquries_FE_train.csv')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-04T04:45:13.931501Z",
     "start_time": "2018-01-04T04:45:12.303827Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>virtual</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2950</td>\n",
       "      <td>1475.000000</td>\n",
       "      <td>1475.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>950.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3758</td>\n",
       "      <td>1879.000000</td>\n",
       "      <td>1879.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>3300</td>\n",
       "      <td>1650.000000</td>\n",
       "      <td>1100.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2</td>\n",
       "      <td>4900</td>\n",
       "      <td>1633.333333</td>\n",
       "      <td>1633.333333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 227 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.0         1   2950      1475.000000     1475.000000        0.0   \n",
       "1        1.0         2   2850      1425.000000      950.000000       -1.0   \n",
       "2        1.0         1   3758      1879.000000     1879.000000        0.0   \n",
       "3        1.0         2   3300      1650.000000     1100.000000       -1.0   \n",
       "4        2.0         2   4900      1633.333333     1633.333333        0.0   \n",
       "\n",
       "   room_num  Year  Month  Day  ...   virtual  walk  walls  war  washer  water  \\\n",
       "0       2.0  2016      6   11  ...         0     0      0    0       0      0   \n",
       "1       3.0  2016      6   24  ...         0     0      0    1       0      0   \n",
       "2       2.0  2016      6    3  ...         0     0      0    0       0      0   \n",
       "3       3.0  2016      6   11  ...         0     0      0    0       0      0   \n",
       "4       4.0  2016      4   12  ...         0     0      0    1       0      0   \n",
       "\n",
       "   wheelchair  wifi  windows  work  \n",
       "0           0     0        0     0  \n",
       "1           0     0        0     0  \n",
       "2           0     0        0     0  \n",
       "3           1     0        0     0  \n",
       "4           0     0        0     0  \n",
       "\n",
       "[5 rows x 227 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target = pd.read_csv(dpath + 'RentListingInquries_FE_test.csv')\n",
    "target.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-04T04:45:14.814358Z",
     "start_time": "2018-01-04T04:45:14.604513Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/pandas/core/indexing.py:517: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self.obj[item] = s\n"
     ]
    }
   ],
   "source": [
    "def remove_noise(df):\n",
    "#remove some noise\n",
    "    df= df[df.price < 10000]\n",
    "\n",
    "    df.loc[df[\"bathrooms\"] == 112, \"bathrooms\"] = 1.5\n",
    "    df.loc[df[\"bathrooms\"] == 10, \"bathrooms\"] = 1\n",
    "    df.loc[df[\"bathrooms\"] == 20, \"bathrooms\"] = 2\n",
    "    return df\n",
    "data = remove_noise(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 特征编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-04T04:45:16.774681Z",
     "start_time": "2018-01-04T04:45:16.711672Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_train = data['interest_level']\n",
    "X_train = data.drop('interest_level',axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-04T04:45:18.694991Z",
     "start_time": "2018-01-04T04:45:18.349050Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 数据标准化 \n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# 初始化特征的标准化器\n",
    "ss_X = StandardScaler()\n",
    "\n",
    "# 分别对训练和测试数据的特征进行标准化处理\n",
    "X_train = ss_X.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 模型训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-04T04:45:20.194770Z",
     "start_time": "2018-01-04T04:45:20.172172Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)\n",
    "kfold = list(kfold.split(X_train,y_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-04T04:45:21.021679Z",
     "start_time": "2018-01-04T04:45:21.013852Z"
    },
    "run_control": {
     "marked": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'learning_rate': [0.001, 0.01, 0.03, 0.05, 0.1]}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learning_rate = [0.001, 0.01, 0.03,0.05, 0.1]\n",
    "\n",
    "param_test6_1 = dict(learning_rate=learning_rate)\n",
    "param_test6_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-04T07:26:40.804014Z",
     "start_time": "2018-01-04T04:45:29.563221Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.65152, std: 0.00378, params: {'learning_rate': 0.001},\n",
       "  mean: -0.58831, std: 0.00486, params: {'learning_rate': 0.01},\n",
       "  mean: -0.60284, std: 0.00603, params: {'learning_rate': 0.03},\n",
       "  mean: -0.62883, std: 0.00665, params: {'learning_rate': 0.05},\n",
       "  mean: -0.69303, std: 0.00893, params: {'learning_rate': 0.1}],\n",
       " {'learning_rate': 0.01},\n",
       " -0.58831001199634103)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb6_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=3000,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=6,\n",
    "        min_child_weight=2,\n",
    "        gamma=0,\n",
    "        subsample=0.9,\n",
    "        colsample_bytree=0.6,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        reg_alpha= 2,\n",
    "        reg_lambda= 0.5,\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch6_1 = GridSearchCV(xgb6_1, param_grid = param_test6_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch6_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch6_1.grid_scores_, gsearch6_1.best_params_,     gsearch6_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-04T07:26:41.080480Z",
     "start_time": "2018-01-04T07:26:41.060866Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([ 2041.62716827,  2084.49221983,  2189.49575028,  2101.50374765,\n",
       "         1893.9300169 ]),\n",
       " 'mean_score_time': array([ 59.13894658,  31.09319162,  52.60287781,  39.03544736,  51.6536231 ]),\n",
       " 'mean_test_score': array([-0.65151555, -0.58831001, -0.60283982, -0.6288345 , -0.6930264 ]),\n",
       " 'mean_train_score': array([-0.63090572, -0.44899379, -0.28787717, -0.20112345, -0.10694892]),\n",
       " 'param_learning_rate': masked_array(data = [0.001 0.01 0.03 0.05 0.1],\n",
       "              mask = [False False False False False],\n",
       "        fill_value = ?),\n",
       " 'params': [{'learning_rate': 0.001},\n",
       "  {'learning_rate': 0.01},\n",
       "  {'learning_rate': 0.03},\n",
       "  {'learning_rate': 0.05},\n",
       "  {'learning_rate': 0.1}],\n",
       " 'rank_test_score': array([4, 1, 2, 3, 5], dtype=int32),\n",
       " 'split0_test_score': array([-0.64456734, -0.58066536, -0.59510289, -0.62083293, -0.68342538]),\n",
       " 'split0_train_score': array([-0.63335766, -0.45157989, -0.28898866, -0.20187744, -0.1072746 ]),\n",
       " 'split1_test_score': array([-0.65578922, -0.59503268, -0.61063144, -0.63651575, -0.70301175]),\n",
       " 'split1_train_score': array([-0.62945293, -0.44724305, -0.28705896, -0.2005259 , -0.10655591]),\n",
       " 'split2_test_score': array([-0.65147007, -0.59110687, -0.60816214, -0.63522147, -0.70293528]),\n",
       " 'split2_train_score': array([-0.63039668, -0.44812606, -0.28718914, -0.20025823, -0.10675726]),\n",
       " 'split3_test_score': array([-0.65362633, -0.58894767, -0.60317887, -0.63022396, -0.69316074]),\n",
       " 'split3_train_score': array([-0.63025088, -0.44802024, -0.28732816, -0.20089568, -0.10681101]),\n",
       " 'split4_test_score': array([-0.65212652, -0.58579916, -0.59712533, -0.62138007, -0.68260081]),\n",
       " 'split4_train_score': array([-0.63107044, -0.44999969, -0.28882095, -0.20205999, -0.10734581]),\n",
       " 'std_fit_time': array([   8.23121259,   89.86398884,    5.32446927,   47.22101196,\n",
       "         388.93661228]),\n",
       " 'std_score_time': array([  6.20301279,   2.01837504,  13.21244349,  15.22092661,  18.00899745]),\n",
       " 'std_test_score': array([ 0.00377845,  0.00486171,  0.00602788,  0.00665322,  0.00893073]),\n",
       " 'std_train_score': array([ 0.00132949,  0.00157896,  0.00084503,  0.00072155,  0.00030783])}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch6_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-04T07:41:00.794680Z",
     "start_time": "2018-01-04T07:41:00.653951Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.588310 using {'learning_rate': 0.01}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAELCAYAAAAoUKpTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl4VOXZx/HvnYSwBkhYZUkCCCIi\nIBlABHdLqVZxq7u1ioJa1L6t1trtrXa39W3rVrViVUBxp6BWXOoCuJGwSpQ1AcIOYYeQ7X7/mKGN\nKZiQzORkkt/nunIxc+ZMzv2YOL+c5znneczdERER+SoJQRcgIiL1n8JCRESqpLAQEZEqKSxERKRK\nCgsREamSwkJERKqksBARkSopLEREpEoKCxERqVJS0AVES/v27T0zMzPoMkRE4kpOTs5Wd+9Q1X4N\nJiwyMzPJzs4OugwRkbhiZqurs5+6oUREpEoKCxERqZLCQkREqqSwEBGRKiksRESkSgoLERGpksJC\nRESqpLAQEYljOasLeSt3U8yP02BuyhMRaUzcnckfr+aeV3Pp3TGFM/t2JCHBYnY8hYWISJwpKinj\nJ698xkvzCjijb0f+dOmgmAYFKCxEROLK2sJ93Dg5hyXrd/G9s3pz6xm9Yx4UoLAQEYkbHyzbwq1T\n51NW7ky8JsSZx3aqs2MrLERE6jl35+H3VvLHN5dyTKcUHrkqi8z2Leu0BoWFiEg9truohNtfWMjM\nJZs4d2AXfn/R8bRIrvuPboWFiEg9tWLzbsZNymH1tn387Jv9uG5EJmaxH584FIWFiEg99M/FG7j9\nhYU0T05kyvXDOLFnu0DrUViIiNQjZeXOH2Yu5ZH3V3JCelsevnIwR7VpHnRZCgsRkfqicG8xtz47\nn9krtnLlsHR+fm4/miYlBl0WoLAQEakXFhfs5MbJOWzZc4B7LxrAJUO6B13SlygsREQC9nz2Wn46\n7TM6tGrKizcOZ0C3tkGX9F8UFiIiASkuLefuGUuY8skaRhzdjgcuH0xay+SgyzokhYWISAA27izi\npik5zF+zgxtP7cXto/qQlFh/JwJXWIiI1LFPVm3ju8/MY39xGQ9fOZizjz8q6JKqpLAQEakj7s4T\nc/L5zeufk5HWgmdvOJHenVKCLqtaFBYiInVgX3Epd728mH8sWM+ofp2475KBpDRrEnRZ1aawEBGJ\nsdXb9jJ+Ug5LN+3mjq8fw02n9qqTacWjSWEhIhJD736xmdumzichwXjy2qGc2qdD0CXViMJCRCQG\nysud+/+1nL+8s5xjO7fm0auz6J7WIuiyakxhISISZTv3l/D95xbwzhebuXBwV35zwfE0a1I/pu2o\nqZhe1Gtmo81sqZmtMLMfHWafS8ws18yWmNkzFbbfG9n2uZndb0HNyysicgS+2LiLMQ/O5v1lW/jl\nmOO471sD4z4oIIZnFmaWCDwEfA0oAOaa2XR3z62wT2/gLmCEu283s46R7ScBI4ABkV1nA6cC78Wq\nXhGR2pq+cD13vriIlGZJTB13IqHMtKBLippYdkMNBVa4+yoAM5sKjAFyK+xzA/CQu28HcPfNke0O\nNAOSAQOaAJtiWKuISI2VlJXzu39+wcTZeQzJTOWhKwbTsXWzoMuKqliGRVdgbYXnBcCwSvv0ATCz\nOUAi8At3f8PdPzKzd4ENhMPiQXf/PIa1iojUyJbdB5jwzDw+ySvkOydl8pNzjqVJPZ62o6ZiGRaH\nGmPwQxy/N3Aa0A2YZWb9gfbAsZFtAG+Z2Snu/sGXDmA2DhgHkJ6eHr3KRUSqYf6a7dw0eR479hfz\np0sHcsEJ3ap+U5yKZfwVABUnZO8GrD/EPv9w9xJ3zwOWEg6PC4CP3X2Pu+8B/gmcWPkA7v6Yu4fc\nPdShQ3xeuywi8cfdeeaTNVz66Mc0STJeuumkBh0UENuwmAv0NrMeZpYMXAZMr7TPNOB0ADNrT7hb\nahWwBjjVzJLMrAnhwW11Q4lI4IpKyrjzpUX8+JXFDO/VjhkTRnJclzZBlxVzMeuGcvdSM5sAzCQ8\nHvGEuy8xs3uAbHefHnltlJnlAmXAHe6+zcxeBM4AFhPuunrD3WfEqlYRkepYt2M/N03OYVHBTm49\n42huO6sPiXE2bUdNmXvlYYT4FAqFPDs7O+gyRKSBmrNiK7c8O5+S0nL+79JBfK1fp6BLigozy3H3\nUFX76Q5uEZGv4O489sEqfv/GF/Tq0IpHr86iZ4dWQZdV5xQWIiKHsedAKT98cSGvL97IOccfxb0X\nD6Bl08b5sdk4Wy0iUoWVW/YwflIOq7bs4SdnH8v1J/egMc86pLAQEalk5pKN/OD5hSQnJTB57DBO\nOrp90CUFTmEhIhJRVu786a1lPPjuCgZ2a8PDV2XRtW3zoMuqFxQWIiLA9r3F3PbcAj5YtoXLhnTn\nF+cd1yBmi40WhYWINHqfrdvJjZNz2LzrAL+98HguH6rpgypTWIhIo/byvALuenkxaS2Tef7G4Qzq\n3jbokuolhYWINErFpeX8+rVcnvpoNSf2TOPBKwbTvlXToMuqtxQWItLobNpVxM1T5pGzejs3nNyD\nO0f3JakBTiseTQoLEWlU5uYXcvOUeew9UMoDl5/AuQO7BF1SXFBYiEij4O48/dFqfvlqLt1SmzN5\n7DCO6ZwSdFlxQ2EhIg3e/uIyfvLKYl6ev46zju3IfZcMok3zJkGXFVcUFiLSoK3Zto/xk3P4YuMu\nvv+1Pkw4/WgSGsm04tGksBCRBuu9pZu5beoC3J0nvjOE04/pGHRJcUthISINTnm58/B7K7jvrWUc\n0ymFR6/OIqNdy6DLimsKCxFpUHYVlfCD5xfyVu4mxgzqwu8uHEDzZE3bUVsKCxFpMJZv2s34STms\nKdzH/57bj++clNmopxWPJoWFiDQIry3awB0vLqRFchLP3HAiQ3ukBV1Sg6KwEJG4VlpWzh9mLuXR\nD1YxOL0tf70qi06tmwVdVoOjsBCRuLVtzwFueXY+H67cxtUnZvCzb/YjOUnTdsSCwkJE4tLCtTu4\naXIOW/cW84eLB/CtUPegS2rQFBYiEneem7uGn01bQoeUprx800n079om6JIaPIWFiMSNA6Vl/GJ6\nLs9+uoaTe7fn/stOILVlctBlNQoKCxGJCxt27ufGyfNYuHYHN5/Wix+MOoZETdtRZxQWIlLvfbRy\nGxOemUdRSRmPXDWY0f2PCrqkRkdhISL1lrszcXYev/3nF2S2a8GjVw/n6I6tgi6rUVJYiEi9tK+4\nlB++uIhXF21g9HGd+eMlA2nVVB9ZQdF/eRGpd/K27uXGSTks37ybO0f35cZTe2rajoDF9O4VMxtt\nZkvNbIWZ/egw+1xiZrlmtsTMnqmwPd3M3jSzzyOvZ8ayVhGpH975fBPnPTibzbuLeOq6odx0Wi8F\nRT0QszMLM0sEHgK+BhQAc81survnVtinN3AXMMLdt5tZxcnmnwZ+7e5vmVkroDxWtYpI8MrLnT+/\ns5z731lO/66teeSqLLqltgi6LImIZTfUUGCFu68CMLOpwBggt8I+NwAPuft2AHffHNm3H5Dk7m9F\ntu+JYZ0iErCd+0r43nPzeXfpFi7O6savzu9PsyaaVrw+iWVYdAXWVnheAAyrtE8fADObAyQCv3D3\nNyLbd5jZy0AP4G3gR+5eFsN6RSQAn2/YxfhJOWzYuZ9fnd+fK4elq9upHoplWBzqp+2HOH5v4DSg\nGzDLzPpHtp8MnACsAZ4DvgNM/NIBzMYB4wDS09OjV7mI1Il/LFjHnS8tok3zJkwdN5ysjNSgS5LD\niOUAdwFQcWavbsD6Q+zzD3cvcfc8YCnh8CgA5rv7KncvBaYBgysfwN0fc/eQu4c6dOgQk0aISPSV\nlJVz94wl3DZ1AQO6tWXGLSMVFPVcLMNiLtDbzHqYWTJwGTC90j7TgNMBzKw94e6nVZH3pprZwQQ4\ngy+PdYhInNq8u4gr//YJf5+Tz3UjejDl+mF0TNH6E/VdzLqh3L3UzCYAMwmPRzzh7kvM7B4g292n\nR14bZWa5QBlwh7tvAzCz24F3LNx5mQP8LVa1ikjdyFm9nZun5LBzfwl/uWwQYwZ1DbokqSZzrzyM\nEJ9CoZBnZ2cHXYaIHIK7M/mTNdwzYwlHtWnOo1dncexRrYMuSwAzy3H3UFX76Q5uEYmpopIyfjrt\nM17MKeD0Yzrw50tPoE2LJkGXJUdIYSEiMVOwfR83Ts7hs3W7uO3M3tx2Zm8SNK14XFJYiEhMzFq+\nhVufnU9pufP4t0Oc1a9T0CVJLSgsRCSq3J2/vr+SP85cSu+OKTxydRY92rcMuiypJYWFiETN7qIS\n7nhhEW8s2ci5A7vw+4uOp0WyPmYaAv0URSQqVmzew/hJ2eRv28dPzzmWsSN7aNqOBkRhISK19sZn\nG7n9hYU0TUpg8thhDO/VLuiSJMoUFiJSY2Xlzh/fXMpf31vJwO5teeSqwRzVpnnQZUkMKCxEpEYK\n9xZz29T5zFq+lSuGpfO/5/ajaZKmFW+ojigszCwBaOXuu2JUTyD2HCjFgJZa31ekWj5bt5Pxk3LY\nsvsAv7/oeC4dolmfG7oqJxI0s2fMrLWZtSQ8md9SM7sj9qXVjYLt+xjwi5lMX1h5QlwROZQXcwq4\n6K8f4u68cONwBUUjUZ1ZZ/tFziTOB14H0oGrY1pVHeratjmpLZLJzt8edCki9VpxaTk/nbaY219Y\nSFZGKjNuGcnA7m2DLkvqSHX6XZqYWRPCYfGgu5eYWcOYfRAwMwZnpJKzujDoUkTqrY07i7h5Sg7z\n1uxg/Ck9uePrx5CUGMsVDqS+qU5YPArkAwuBD8wsA2hQYxZDMlN5K3cTW3YfoENK06DLEalXPs0r\n5OYp89hXXMpDVwzmnAFHBV2SBKDKPw3c/X537+ruZ3vYaiILFjUUWRlpADq7EKnA3fn7nDyu+NvH\ntG6WxLTvjlBQNGLVGeC+LTLAbWY20czmEV65rsHo37U1TZMSNG4hErG/uIzvPbeAu2fkcnrfjkyb\nMII+nVKCLksCVJ1Ox+siA9yjgA7AtcDvYlpVHWualMjAbm2Zu1phIbJ6214ueHgO0xeu5/ZRfXj0\nqixaN9P6E41ddcYsDk7ucjbwd3dfaA1wwpeszFT+9sEq9heX0TxZNxZJ4/Tu0s3c9ux8zIwnrx3K\nqX06BF2S1BPVObPIMbM3CYfFTDNLAcpjW1bdG5KZSmm5s7BgR9CliNS58nLn/neWc92Tc+mW2oIZ\nE0YqKORLqnNmMRYYBKxy931m1o5wV1SDMjg9FYDs/EJO7KlJ0KTx2Lm/hB88v4C3P9/MhSd05dcX\nHK+za/kvVYaFu5ebWTfgikjv0/vuPiPmldWxti2S6d2xFdkat5BGZOnG3YyflE3B9v3cfd5xfHt4\nhqYVl0OqMizM7HfAEGBKZNOtZnaSu98V08oCEMpM49VF6ykvd60TLA3ejIXr+eGLi2jVLImp404k\nlJkWdElSj1VnzOJs4Gvu/oS7PwGMBs6JbVnBCGWksruolGWbdwddikjMlJaV8+vXcrnl2fkc16U1\nr90yUkEhVaruNKttgYN3rLWJUS2BC2UeHLfYTt/OrQOuRiT6tu45wIRn5vHxqkKuGZ7BT87pR3KS\npu2QqlUnLH4LzDezdwlfRnsK0OC6oADS01rQIaUp2fmFXHViRtDliETV/DXbuXnKPAr3FvN/lwzk\nwsHdgi5J4kh1BrifNbP3CI9bGHAn1eu+ijtmRigjVYPc0uA8++ka/vcfS+jYuikv3XQS/bs22A4C\niZFqdUO5+wZg+sHnZraG8FTlDU4oM41/fraRjTuL6NymWdDliNRKUUkZv5i+hKlz13JKnw785dJB\npLZMDrosiUM1PUNosJcKhTIi4xaaVFDi3Pod+7n00Y+YOnctE04/mr9/Z4iCQmqspuuINpj1LCrr\n16U1zZskkp2/nW8O6BJ0OSI18uGKrUx4dj7FpeU8dnUWo47rHHRJEucOGxZm9gCHDgUjfHVUlcxs\nNPAXIBF43N3/awJCM7sE+EXkWAvd/YoKr7UGPgdecfcJ1TlmbTVJTGBQ97Y6s5C45O78bdYqfvfP\nL+jZoRWPXp1Frw6tgi5LGoCvOrPIruFrAJhZIvAQ8DWgAJhrZtPdPbfCPr0JX1k1wt23m1nHSt/m\nl8D7VR0r2kKZqTz83kr2HiilZdOannyJ1K29B0r54YuLeG3xBs4+vjP3XjyQVvr9lSg57G+Suz9V\ny+89FFjh7qsAzGwqMAbIrbDPDcBD7r49cszNB18wsyygE/AGEKplLUckKyOVsnJnwdodjDi6fV0e\nWqRGVm3Zw/hJOazcsoe7vtGXcaf01LQdElWxvAS2K7C2wvOCyLaK+gB9zGyOmX0c6bbCzBKA+4A7\nvuoAZjbOzLLNLHvLli1RK3xwRipmMDdfXVFS/725ZCNjHpzDtr3FTBo7jPGn9lJQSNTF8hz1UL+t\nlcdAkoDewGlAN2CWmfUHrgJed/e1X/VL7+6PAY8BhEKhqA26t27WhGM6pZCj+y2kHisrd/789jIe\n+NcKBnRrw1+vyqJr2+ZBlyUNVCzDogDoXuF5N2D9Ifb52N1LgDwzW0o4PIYDJ5vZzUArINnM9rj7\nj2JY75eEMlN5Zd46SsvKSUpskPcgShzbsa+Y26Yu4P1lW7gk1I17xvSnWRNNKy6xU51ZZ+8/xOad\nQLa7/+Mr3joX6G1mPYB1wGXAFZX2mQZcDjxpZu0Jd0utcvcrKxz/O0CoLoMCYEhmGpM/XsMXG3fr\nblepV5as38mNk3PYuLOI31xwPJcP7a5uJ4m56vzJ3Izw4kfLI18DgDRgrJn9+XBvcvdSYAIwk/Dl\nr8+7+xIzu8fMzovsNhPYZma5wLvAHe6+rcatiaKsyM156oqS+uSV+QVc+PCHlJQ6z48fzhXD0hUU\nUifM/au7+s3sX8CoyIc/ZpYEvEn4ktjF7t4v5lVWQygU8uzsKq/orTZ356Tf/YtQZhoPXH5C1L6v\nSE2UlJXz69c+58kP8xnWI40HrxhMh5SmQZclDYCZ5bh7lVecVmfMoivQknDXE5HHXdy9zMwO1KLG\nes3MyMpIJVtXREnANu8q4rvPzGNu/nauH9mDO7/RlyYaR5M6Vp2wuBdYEJl59uAU5b8xs5bA2zGs\nLXChjFReXbSBdTv26yoTCUTO6kJumjyP3UWl3H/5CZw3UFPQSDCqM0X5RDN7nfBNdgb82N0PXtX0\nlfdBxLuDq4dl5xfSdVDlW0REYsfdmfTxau6ZkUu31OY8PXaoFuSSQFX30tkhwMmRx2X89yWwDVLf\nzim0TA5PKjhGYSF1pKikjB+/spiX563jzL4d+b9LB9GmeZOgy5JGrjqXzv6OcFhMiWy61cxOcvcG\nuVpeRUmJCZyQrsWQpO6sLdzHjZNzyN2wi/85qw+3nHE0CQm62kmCV50zi7OBQe5eDmBmTwHzaaBL\nq1YWykzlL+8sZ1dRCa2b6a87iZ0Plm3h1qnzKS93Jl4T4oy+nYIuSeTfqntJRcUpyRvVHWqhjDTc\nYf6aHUGXIg1Uebnz0LsruObvn9K5dTNm3DJSQSH1TnXOLH4LzDezd/nP1VCN4qwCYFB6WxIMcvIL\nObVPh6DLkQZmd1EJP3h+IW/mbmLMoC789sLjaZGsacWl/qnO1VDPRi6bHUI4LO4ktrPV1iutmibR\nr0tr5uZr3EKia/mm3YyfnMPqbfv4+Tf7ce2ITN2NLfVWtf6EcfcNwPSDz81sDZAeq6Lqm1BGGs/N\nXUtJWbluhpKo+OfiDdz+wkKaJycy5fphnNizXdAliXylmn7yNao/f0KZqewvKSN3/a6gS5E4V1pW\nzm//+Tk3TZlHn84pvHrLyQoKiQs17RyN2toR8SCUEbk5b/V2Bnav1vLjIv+lcG8xtzw7jzkrtnHV\nien87Jv9aJqkacUlPhw2LMzsAQ4dCsaXr45q8Dq3aUbXts3JWV3I2JE9gi5H4tCigh3cNHkeW/Yc\n4N6LB3BJqHvVbxKpR77qzOKrpnCN3vSucWJIZipzVm7D3TUIKUfk+blr+ek/PqNDq6a8dONJHN+t\nUV19Lg3EYcPC3Z+qvM3MOrv7xtiWVD9lZaYxbcF61hbuJ71di6DLkThwoLSMu2fk8swnaxh5dHvu\nv/wE0lomB12WSI0c6ZjF68DgWBRS34UiiyFlry5UWEiVNu0q4sbJOcxfs4ObTuvF7aOOIVHTdkgc\nO9KroRrtb3ufTimkNEvS/RZSpez8Qr75wGyWbtzNw1cO5s7RfRUUEveO9MzibzGpIg4kJhiD01PJ\nWa3FkOTQ3J0pn6zh7hlL6Nq2OVOuH0afTilBlyUSFUd0ZuHuD8eqkHgQykhl2aY97NxXEnQpUs8c\nKC3jRy8t5qfTPmPE0e35x3dHKiikQdHtyEfg4GJIOWt0diH/sXFnEZc++jHPZa/lljOOZuI1Q2jT\nQjMUS8OiGcuOwKDubUlKMLLzt2tWUAHg07xCbp4yj/3FpTxyVRaj+3cOuiSRmFBYHIHmyYkc16U1\n2RrkbvQqLnuantaCZ28YRm91O0kDprA4QqHMNCZ/vJri0nKSk9SL1xgVlZTxs2mf8UJOAWf27cif\nLhukhbGkwdOn3REKZaRyoLScz9bvDLoUCcD6Hfu55NGPeCGngFvP7M3fvh1SUEijoDOLI5SVGbk5\nL7+QwempAVcjdemTVdv47jPzKCop57Grsxh1nMYnpPHQmcUR6pjSjIx2LTRu0Yi4O0/OyePKxz+h\ndfMmTPvuCAWFNDo6s6iBrIxU3l+6RZMKNgJFJWX8+JXFvDxvHWcd24k/XTqQFHU7SSOkM4saGJKZ\nxra9xeRt3Rt0KRJD63bs51uPfMTL89bxP2f14bGrsxQU0mjFNCzMbLSZLTWzFWb2o8Psc4mZ5ZrZ\nEjN7JrJtkJl9FNm2yMwujWWdR+o/kwqqK6qh+mjlNs59YDb5W/fy+LdD3HZWbxI0v5M0YjHrhjKz\nROAh4GtAATDXzKa7e26FfXoDdwEj3H27mXWMvLQP+La7LzezLkCOmc109x2xqvdI9OrQijbNm5Cd\nX6hFbBoYd+fvc/L59eufk9muBY99O0SvDq2CLkskcLEcsxgKrHD3VQBmNhUYA+RW2OcG4CF33w7g\n7psj/y47uIO7rzezzUAHoF6ERUKCEcpI1ZlFA7O/ODw+8cr8dYzq14n7LtH4hMhBseyG6gqsrfC8\nILKtoj5AHzObY2Yfm9noyt/EzIYCycDKmFVaA1mZqazaspdtew4EXYpEwdrCfVz8yIdMW7CO20f1\n4ZGrND4hUlEszywO1cFbeU3vJKA3cBrQDZhlZv0PdjeZ2VHAJOAady//rwOYjQPGAaSnp0ev8moI\nZUQmFVy9XZdRxrkPV2zlu8/Mo7TcmXhNSPN+iRxCLM8sCoCKHfrdgPWH2Ocf7l7i7nnAUsLhgZm1\nBl4DfuruHx/qAO7+mLuH3D3UoUOHqDfgqwzo1obkxARy1BUVt9ydx2et4qqJn9C+VVOmTxipoBA5\njFiGxVygt5n1MLNk4DJgeqV9pgGnA5hZe8LdUqsi+78CPO3uL8Swxhpr1iSR/l1ba9wiTu0vLuO2\nqQv41Wuf8/XjOvPKd0fQo33LoMsSqbdiFhbuXgpMAGYCnwPPu/sSM7vHzM6L7DYT2GZmucC7wB3u\nvg24BDgF+I6ZLYh8DYpVrTU1JDONxQU7KSopC7oUOQJrC/dx4V8/ZMai9dzx9WN4+MrBtGqq+1NF\nvoq5Vx5GiE+hUMizs7Pr9JhvLtnIuEk5vHDjcIZEFkaS+m328q1MeHYe5eXO/ZefwGnHdKz6TSIN\nmJnluHuoqv10B3ctZB28OU/zRNV77s5jH6zk2098QqeUZkyfMFJBIXIEdO5dC+1aNaVnh5Zk5xcC\nvYIuRw5jX3EpP3xxEa8u2sA5xx/FvRcPoKW6nUSOiP6PqaVQRipv5m6ivNw1HUQ9tGbbPsZNymbZ\npt386Bt9GX9KT03+KFID6oaqpVBGGjv2lbByy56gS5FK3l+2hXMfnM2GnUU8ee1Qbjy1l4JCpIYU\nFrUUytSkgvWNu/PX91Zy7d8/5ag2zZgxYSSn9Knb+3BEGhqFRS31aN+Sdi2TNchdT+w9UMqEZ+bz\n+ze+4Ozjj+Llm08ivV2LoMsSiXsas6glM2NwRirZqwuDLqXRy9+6l/GTcli+eTc/PrsvN5ys8QmR\naFFYRMGQzFTeyt3Elt0H6JDSNOhyGqV3l27mtmfnk5BgPH3dMEb2bh90SSINirqhoiDr35MK6uyi\nrrk7D727guuenEu31BbMmDBSQSESAwqLKOjftTVNkxKYq3GLOrXnQCk3TZ7HH2Yu5dwBXXjpppPo\nnqbxCZFYUDdUFDRNSmRgt7a6IqoO5W3dy7ins1m5ZQ8/PedYxo7sofEJkRjSmUWUZGWmsmTdTvYX\na1LBWPvXF5s478HZbN1zgMljh3G9BrJFYk5hESVDMlMpLXcWrK0XK782SOXlzgPvLGfsU9mkp7Vg\nxi0jOelojU+I1AV1Q0XJ4PTwzXk5qwsZ3qtdwNU0PHsOlPL95xbwZu4mLjihK7+98HiaNUkMuiyR\nRkNhESVtWyTTu2MrDXLHwMotexg/KYe8rXv5+Tf7ce2ITHU7idQxhUUUhTLTeHXRek0qGEVv527i\nf55bQJOkBCaPHaazNpGAaMwiikIZqewuKmXZ5t1BlxL3ysudP7+9jOufziazfUtm3DJSQSESIJ1Z\nRNHBSQXn5m+nb+fWAVcTv3YVlfD95xby9uebuHBwV35zgcYnRIKmM4soSk9rQYeUpuTk607umlqx\neQ/nPzSH95Zu5u7zjuO+bw1UUIjUAzqziCIzI5SRqpvzaujNJRv5/vMLaZqUwJTrhzGsp7qdROoL\nnVlEWSgzjYLt+9m4syjoUuJGebnzf28tY9ykHHp1CI9PKChE6heFRZSFMg4uhqSuqOrYub+EG57O\n5v53lvOtrG48N344Xdo2D7osEalEYRFl/bq0pnmTRC2GVA3LN+3m/Ifm8P6yLfxyzHHce/EAjU+I\n1FMas4iyJokJDOreVmcWVXjjs4384PkFNE9O4tlxJzIkMy3okkTkK+jMIgZCmankrt/FngOlQZdS\n75SVO/e9uZQbJ+dwdKcUZtwDypz8AAAOZUlEQVQyQkEhEgcUFjGQlZFKucOCNZpUsKKd+0u4/qm5\nPPCvFVwa6s7z40/kqDYanxCJB+qGioHBGamYhQe5tWpb2NKNuxk/KZt1O/bz6wv6c8XQdM3vJBJH\nFBYx0LpZE47plEKO7rdgUcEOHp+Vx2uLN5DWMpmp40789zK0IhI/FBYxEspM5ZV56ygtKycpsXH1\n9pWVO29/vomJs/L4NL+QlKZJjB3Zg+tP7kHHlGZBlyciNRDTTzEzG21mS81shZn96DD7XGJmuWa2\nxMyeqbD9GjNbHvm6JpZ1xsKQzDT2FpfxxcbGM6ngvuJSnv4onzPve4/xk3JYv3M/P/tmPz768Zn8\n+OxjFRQicSxmZxZmlgg8BHwNKADmmtl0d8+tsE9v4C5ghLtvN7OOke1pwP8CIcCBnMh746ZfJyvj\n4GJI2+nftU3A1cTWxp1FPPVRPs98soad+0sYnN6WH47uy6h+nRrdWZVIQxXLbqihwAp3XwVgZlOB\nMUBuhX1uAB46GALuvjmy/evAW+5eGHnvW8Bo4NkY1htVXds256g2zZibX8g1J2UGXU5MfLZuJ0/M\nzmP6wvWUuzO6f2fGjuz576AUkYYjlmHRFVhb4XkBMKzSPn0AzGwOkAj8wt3fOMx7u8au1OgzM7Iy\nUsnO3467N5grf8rLnXeXbubxWXl8tGobLZMT+fbwTK4dkUn3tBZBlyciMRLLsDjUp6Mf4vi9gdOA\nbsAsM+tfzfdiZuOAcQDp6em1qTUmQhmpvLpoA+t27Kdbanx/kO4vLuOleQU8MTuPVVv30qVNM358\ndl8uG5pO62ZNgi5PRGIslmFRAHSv8LwbsP4Q+3zs7iVAnpktJRweBYQDpOJ736t8AHd/DHgMIBQK\n/VeYBC0UuTM5Z/X2uA2LzbuLmPTRaiZ/vJrt+0oY0K0N919+At/o35kmGo8QaTRiGRZzgd5m1gNY\nB1wGXFFpn2nA5cCTZtaecLfUKmAl8BszO9j5PYrwQHhc6ds5hZbJ4UkFxwyKq140Pt+wi4mz85i+\nYD0l5eWM6teJ60/uSSgjtcF0qYlI9cUsLNy91MwmADMJj0c84e5LzOweINvdp0deG2VmuUAZcIe7\nbwMws18SDhyAew4OdseTpMQETkhPZW6crJzn7ry/bAuPz8pj9oqtNG+SyOVDu3PtiB5ktm8ZdHki\nEqCY3pTn7q8Dr1fa9vMKjx34fuSr8nufAJ6IZX11IZSZyl/eWc6uopJ627dfVFLGtPnrmDg7j+Wb\n99CpdVPuHN2XK4am06ZF/axZROqW7uCOsVBGGu4wf80OTu3TIehyvmTrngP/Ho/YtreYfke15k+X\nDuSc47uQnKTxCBH5D4VFjA1Kb0uCQXZ+Yb0Ji+WbdjNxdh4vz19HcWk5Zx3bkbEje3JizzSNR4jI\nISksYqxV0yT6dWkd+Mp57s7sFVt5fFYe7y/bQrMmCXwrqxvXjexBrw6tAq1NROo/hUUdCGWkMXXu\nGkrKyuv8ctMDpWVMX7CeibPz+GLjbjqkNOX2UX24YlgGaS2T67QWEYlfCos6kJWRypMf5pO7fhcD\nu7etk2MW7i1mysereeqj1Wzdc4C+nVP447cGcu7Ao2iapHWuReTIKCzqQCgzfLtI9urtMQ+LFZv3\n8MScPF7KKeBAaTmnHdOB60f2ZMTR7TQeISI1prCoA0e1aU7Xts3Jzi9k7MgeUf/+7s5Hq7YxcVYe\n73yxmeSkBC48oSvXjexBn04pUT+eiDQ+Cos6MiQzlTkrt0V1UsHi0nJeXbSex2flkbthF+1aJvO9\ns3pz1YkZtG/VNCrHEBEBhUWdycpMY9qC9awt3E96u9rNE7VjXzFTPlnDUx/ms3n3AXp3bMXvLzqe\nMYO60qyJxiNEJPoUFnUkFFnjYW5+YY3DIm/rXv4+J48XsgvYX1LGyb3bc+/FAzi1TweNR4hITCks\n6kifTimkNEsie/V2LsrqVu33uTuf5hXy+Ow83v58E00SEhgzqAtjT+5B386tY1ixiMh/KCzqSGKC\nMTg9lZzV1ZtUsKSsnNcXb2Di7DwWFewktUUTJpx+NFcPz9Ba1iJS5xQWdSiUkcp9b21hx75i2rY4\n9A1xO/eXMPXTNTz5YT4bdhbRs31LfnV+fy4a3I3myRqPEJFgKCzq0MHFkOat2c4ZfTt96bW1hfuY\nODuP57PXsq+4jOE92/Gr8/tz+jEdSUjQeISIBEthUYcGdW9LUoIxN/8/YZGzupDHZ+Uxc8lGEsw4\nb2AXrhvZg/5d2wRcrYjIfygs6lDz5ESO69KaT/MKeW3RBh6fvYr5a3bQulkS40/txTXDM+ncRuMR\nIlL/KCzqWCgzjYmz88hZvZ2Mdi24Z8xxXDS4Gy2b6kchIvWXPqHq2CWh7mzcWcR5g7pw1rGdSNR4\nhIjEAYVFHTumcwoPXTk46DJERI6I1s4UEZEqKSxERKRKCgsREamSwkJERKqksBARkSopLEREpEoK\nCxERqZLCQkREqmTuHnQNUWFmW4DVR/i29sDWGJRTnzXGNkPjbHdjbDM0znbXps0Z7t6hqp0aTFjU\nhJllu3so6DrqUmNsMzTOdjfGNkPjbHddtFndUCIiUiWFhYiIVKmxh8VjQRcQgMbYZmic7W6MbYbG\n2e6Yt7lRj1mIiEj1NPYzCxERqYYGGRZmNtrMlprZCjP70SFeb2pmz0Ve/8TMMiu8dldk+1Iz+3pd\n1l1bNW23mX3NzHLMbHHk3zPquvaaqs3POvJ6upntMbPb66rmaKjl7/gAM/vIzJZEfuZxsZZvLX6/\nm5jZU5G2fm5md9V17bVRjXafYmbzzKzUzC6u9No1ZrY88nVNrQpx9wb1BSQCK4GeQDKwEOhXaZ+b\ngUcijy8Dnos87hfZvynQI/J9EoNuUx20+wSgS+Rxf2Bd0O2JdZsrvP4S8AJwe9DtqaOfdRKwCBgY\ned4uHn7Ha9nmK4CpkcctgHwgM+g2RbHdmcAA4Gng4grb04BVkX9TI49Ta1pLQzyzGAqscPdV7l4M\nTAXGVNpnDPBU5PGLwJlmZpHtU939gLvnASsi3y8e1Ljd7j7f3ddHti8BmplZ0zqpunZq87PGzM4n\n/D/QkjqqN1pq0+5RwCJ3Xwjg7tvcvayO6q6N2rTZgZZmlgQ0B4qBXXVTdq1V2W53z3f3RUB5pfd+\nHXjL3QvdfTvwFjC6poU0xLDoCqyt8Lwgsu2Q+7h7KbCT8F9Y1XlvfVWbdld0ETDf3Q/EqM5oqnGb\nzawlcCdwdx3UGW21+Vn3AdzMZka6Ln5YB/VGQ23a/CKwF9gArAH+6O6FsS44SmrzmRTVz7OGuAa3\nHWJb5Uu+DrdPdd5bX9Wm3eEXzY4Dfk/4r894UJs23w38yd33RE404klt2p0EjASGAPuAd8wsx93f\niW6JUVebNg8FyoAuhLtjZpnZ2+6+KrolxkRtPpOi+nnWEM8sCoDuFZ53A9Yfbp/IqWkboLCa762v\natNuzKwb8ArwbXdfGfNqo6M2bR4G3Gtm+cD3gB+b2YRYFxwltf0df9/dt7r7PuB1YHDMK6692rT5\nCuANdy9x983AHCBepgOpzWdSdD/Pgh7AicGAUBLhfuge/GdA6LhK+3yXLw+EPR95fBxfHuBeRRwM\n/kWh3W0j+18UdDvqqs2V9vkF8TXAXZufdSowj/BAbxLwNnBO0G2KcZvvBP5O+C/tlkAuMCDoNkWr\n3RX2fZL/HuDOi/zMUyOP02pcS9D/MWL0H/hsYBnhqwh+Etl2D3Be5HEzwlfArAA+BXpWeO9PIu9b\nCnwj6LbURbuBnxLu011Q4atj0O2J9c+6wveIq7CobbuBqwgP6n8G3Bt0W2LdZqBVZPuSSFDcEXRb\notzuIYTPIvYC24AlFd57XeS/xwrg2trUoTu4RUSkSg1xzEJERKJMYSEiIlVSWIiISJUUFiIiUiWF\nhYiIVElhISIiVVJYSKNgZnvq4BjnHWoK6bpgZuebWb8gji2Ng+6zkEbBzPa4e6sofJ9ED2iW1q86\ntpk9Cbzq7i/WbVXSWOjMQhodM7vDzOaa2SIzu7vC9mmRxZ+WmNm4Ctv3mNk9ZvYJMNzM8s3s7sis\nrYvNrG9kv++Y2YORx0+a2f1m9qGZrTq4KI2ZJZjZw5FjvGpmr1desKZSrflm9nMzmw18y8xuiNS+\n0MxeMrMWZnYScB7wBzNbYGa9Il9vRNoz62CNIjWlsJBGxcxGAb0Jz0Q6CMgys1MiL1/n7lmEJ5m7\n1cwOTt/eEvjM3Ye5++zItq3uPhj4K3C4VfaOIjzD6zeB30W2XUh4sZrjgeuB4dUou8jdR7r7VOBl\ndx/i7gOBz4Gx7v4hMJ3wNBaDPDwR5GPALZH23A48XI3jiBxWQ5yiXOSrjIp8zY88b0U4PD4gHBAX\nRLZ3j2zfRnh665cqfZ+XI//mEA6AQ5nm7uVArpl1imwbCbwQ2b7RzN6tRs3PVXjc38x+RXjyx1bA\nzMo7m1kr4CTghQrTr8fDYlZSjykspLEx4Lfu/uiXNpqdBpwFDHf3fWb2HuGJ6SD8l33lsYKDi0OV\ncfj/jyouIGWV/j0Seys8fhI4390Xmtl3gNMOsX8CsMPdB9XgWCKHpG4oaWxmAtdF/vrGzLqaWUfC\nax9sjwRFX+DEGB1/NnBRZOyiE4f+sP8qKcAGM2sCXFlh++7Ia7j7LiDPzL4FYGEDa125NGoKC2lU\n3P1N4BngIzNbTHjJzRTgDSDJzBYBvwQ+jlEJLxGeTvoz4FHgE8LLf1bXzyLveQv4osL2qcAdZjbf\nzHoRDpKxZraQ8NTclderFjkiunRWpI6ZWSsPL+fajvC6CyPcfWPQdYl8FY1ZiNS9V82sLeGVz36p\noJB4oDMLkXrAzF4hvHRmRXe6+39d7SQSBIWFiIhUSQPcIiJSJYWFiIhUSWEhIiJVUliIiEiVFBYi\nIlKl/wfAtRn+MtbenwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1154a7b70>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch6_1.best_score_, gsearch6_1.best_params_))\n",
    "test_means = gsearch6_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch6_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch6_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch6_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pyplot.plot(learning_rate, -test_means)\n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'learning_rate' )                                                                                                      \n",
    "pyplot.ylabel( '-Log Loss' )\n",
    "pyplot.savefig( 'learning_rate.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "当 'learning_rate' =  0.01,logless = 0.588310 比 当 'learning_rate' =  0.1 小一点"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "nbTranslate": {
   "displayLangs": [
    "*"
   ],
   "hotkey": "alt-t",
   "langInMainMenu": true,
   "sourceLang": "en",
   "targetLang": "fr",
   "useGoogleTranslate": true
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
